# -*- coding: UTF-8 -*-

from db import Mongodb
from redisdb import RedisDb



class DuplicatesPipeline:
    "根据传入的数据库名字,集合名字对数据做去重处理"

    def __init__(self,db_name,collection_name):
        "初始化数据"
        self.collection = Mongodb().connect(db_name,collection_name)

        self.redis_db = RedisDb()
        self.redis_db.init_data(self.collection.find())

    def process_item(self, item):
        "使用redis去重"
        url = item['article_url']
        if self.redis_db.get(url):
            print('已包含此目录')
        else:
            try:
                self.collection.insert_one(item)
                self.redis_db.set(url, True)
                print('insert one record succeed!')
            except:
                print('插入一条记录失败:url=%s' %url)

        pass

    def remove_item_from_db(self,item):
        "处理失败,从mongodb中删除插入的数据"
        print(item)
        self.collection.find_one_and_delete({'url':item['article_url']})
        pass


